# 打猴子补丁
from gevent import monkey
monkey.patch_all()
#导入协程池
from gevent.pool import Pool
import schedule
import time
from 代理.setting import PROXIES_SPIDERS,RUN_TIME
import importlib
from 代理.core.verify.http_verify import check_proxy
from 代理.core.store.mysql_store import MONGOPOOL
from 代理.untils.log import logger


class RUNSPIDER():
    def __init__(self):
        self.mongo=MONGOPOOL()
        #创建协程池对象
        self.coroutine_pool=Pool()
    def get_spider_from_settings(self):
        for full_lists in PROXIES_SPIDERS:
            module_name,class_name=full_lists.rsplit('.',maxsplit=1)
            #导入模块
            module=importlib.import_module(module_name)
            #根据类名，从模块中获取类
            cls=getattr(module,class_name)
            #创建代理对象
            spiders=cls()
            yield spiders


    def run(self):
        spiders=self.get_spider_from_settings()
        for spider in spiders:
            # self.__execute__one_spider_task(spider)
            #异步调用
            self.coroutine_pool.apply_async(self.__execute__one_spider_task,args=(spider,))
        self.coroutine_pool.join()

    def __execute__one_spider_task(self, spider):
        #把处理一个代理的代码抽到一个方法
        #用于处理一个代理任务
        try:
            for proxy in spider.get_proxies():
                # 检测代理ip
                proxy = check_proxy(proxy)
                print(proxy)
                if proxy.speed != -1:
                    self.mongo.insert_proxy(proxy)
        except Exception as ex:
            logger.exception(ex)

    @classmethod
    def start(cls):
        #调用run方法
        daili = RUNSPIDER()
        daili.run()
        #利用schedule，间隔执行run方法
        schedule.every(RUNSPIDER).days.do(daili.run)
        while True:
            schedule.run_pending()
            time.sleep(1)

if __name__ == '__main__':
    RUNSPIDER.start()
